{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.028032Z",
     "start_time": "2025-06-22T09:18:46.022200Z"
    }
   },
   "source": [
    "import os\n",
    "# 方法1: 设置环境变量禁用 TorchDynamo\n",
    "os.environ['TORCH_DYNAMO_DISABLE'] = '1'\n",
    "\n",
    "# 导入 torch\n",
    "import torch\n",
    "print(f\"PyTorch 版本: {torch.__version__}\")\n",
    "\n",
    "# 方法2: 在代码中禁用 TorchDynamo\n",
    "try:\n",
    "    if hasattr(torch, '_dynamo'):\n",
    "        torch._dynamo.config.suppress_errors = True\n",
    "        torch._dynamo.config.disable = True\n",
    "        print(\"✓ 已禁用 TorchDynamo\")\n",
    "except Exception as e:\n",
    "    print(f\"⚠ 设置 TorchDynamo 配置时出现警告: {e}\")\n",
    "\n",
    "# 现在可以正常导入其他 PyTorch 模块\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "print(\"✓ PyTorch 导入成功，可以正常使用了！\")\n",
    "\n",
    "# # 导入pytorch，如果你能成功运行说明运行环境已经配置好了~\n",
    "# import torch\n",
    "# print(torch.__version__)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch 版本: 2.7.1+cpu\n",
      "✓ 已禁用 TorchDynamo\n",
      "✓ PyTorch 导入成功，可以正常使用了！\n"
     ]
    }
   ],
   "execution_count": 30
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.049336Z",
     "start_time": "2025-06-22T09:18:46.045592Z"
    }
   },
   "source": [
    "import codecs # 用于读取文本\n",
    "import numpy as np # 用于矩阵计算\n",
    "from tqdm import tqdm # 用于进度条打印"
   ],
   "outputs": [],
   "execution_count": 31
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据读取"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.105843Z",
     "start_time": "2025-06-22T09:18:46.061748Z"
    }
   },
   "source": [
    "# 读取训练集\n",
    "train_lines = codecs.open('input/train.conll').readlines()\n",
    "\n",
    "# 读取验证集\n",
    "dev_lines = codecs.open('input/dev.conll').readlines()\n",
    "\n",
    "# 读取测试集\n",
    "test_lines = codecs.open('input/final_test.txt').readlines()\n",
    "\n",
    "# 查看训练集中的前十个字符\n",
    "# 这里左侧是输入的字符，右侧是字符对应的标签\n",
    "train_lines[:20]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['浙 B-prov\\n',\n",
       " '江 E-prov\\n',\n",
       " '杭 B-city\\n',\n",
       " '州 I-city\\n',\n",
       " '市 E-city\\n',\n",
       " '江 B-district\\n',\n",
       " '干 I-district\\n',\n",
       " '区 E-district\\n',\n",
       " '九 B-town\\n',\n",
       " '堡 I-town\\n',\n",
       " '镇 E-town\\n',\n",
       " '三 B-community\\n',\n",
       " '村 I-community\\n',\n",
       " '村 E-community\\n',\n",
       " '一 B-poi\\n',\n",
       " '区 E-poi\\n',\n",
       " '\\n',\n",
       " '浙 B-prov\\n',\n",
       " '江 I-prov\\n',\n",
       " '省 E-prov\\n']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 32
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.122769Z",
     "start_time": "2025-06-22T09:18:46.118848Z"
    }
   },
   "source": [
    "# 查看测试集两个句子\n",
    "[x.strip().split('\\x01')[1] for x in test_lines[:2]]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['朝阳区小关北里000-0号', '朝阳区惠新东街00号']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 33
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "['浙 B-prov\\n',\n",
    " '江 E-prov\\n',\n",
    " '杭 B-city\\n',\n",
    " '州 I-city\\n',\n",
    " '市 E-city\\n',\n",
    " '江 B-district\\n',\n",
    " '干 I-district\\n',\n",
    " '区 E-district\\n',\n",
    " '九 B-town\\n',\n",
    " '堡 I-town\\n',\n",
    " '镇 E-town\\n',\n",
    " '三 B-community\\n',\n",
    " '村 I-community\\n',\n",
    " '村 E-community\\n',\n",
    " '一 B-poi\\n',\n",
    " '区 E-poi\\n',\n",
    " '\\n']\n",
    " \n",
    "为什么需要拼接呢？因为原始训练集是字符级别的，但模型输入应该以句子级别\n",
    "原始输入句子：浙江杭州市江干区九堡镇三村村一区"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.397680Z",
     "start_time": "2025-06-22T09:18:46.139774Z"
    }
   },
   "source": [
    "# 接下来我们将拼接训练集，原始训练集是对单个字符\n",
    "# train_str 为训练集字符串\n",
    "# train_label 为训练集对应的标签\n",
    "train_str, train_label = [''], [[]]\n",
    "\n",
    "# 从训练集文本中依次读取文本\n",
    "for line in train_lines:\n",
    "    # 如果训练集行中存在空格，则表示格式为 【字符 标注】\n",
    "    # 将当前的字符和标注累计\n",
    "    if ' ' in line:\n",
    "        word, label = line.strip().split(' ')\n",
    "        train_str[-1] += word\n",
    "        train_label[-1].append(label)\n",
    "    \n",
    "    # 否则就是一个句子结束，也就是'\\n'的情况\n",
    "    # 新增一个空的句子和标签\n",
    "    else:\n",
    "        train_str.append('')\n",
    "        train_label.append([])\n",
    "\n",
    "# 输出训练集第一个样本的文本和标签\n",
    "train_str[0], train_label[0]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('浙江杭州市江干区九堡镇三村村一区',\n",
       " ['B-prov',\n",
       "  'E-prov',\n",
       "  'B-city',\n",
       "  'I-city',\n",
       "  'E-city',\n",
       "  'B-district',\n",
       "  'I-district',\n",
       "  'E-district',\n",
       "  'B-town',\n",
       "  'I-town',\n",
       "  'E-town',\n",
       "  'B-community',\n",
       "  'I-community',\n",
       "  'E-community',\n",
       "  'B-poi',\n",
       "  'E-poi'])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 34
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.426277Z",
     "start_time": "2025-06-22T09:18:46.406330Z"
    }
   },
   "source": [
    "# dev_str 为验证集字符串\n",
    "# dev_label 为验证集对应的标签\n",
    "dev_str, dev_label = [''], [[]]\n",
    "\n",
    "# 按照训练集标签的格式，得到验证集输入的句子和对应的标签\n",
    "for line in dev_lines:\n",
    "\n",
    "    if ' ' in line:\n",
    "        word, label = line.strip().split(' ')\n",
    "        dev_str[-1] += word\n",
    "        dev_label[-1].append(label)\n",
    "    else:\n",
    "        dev_str.append('')\n",
    "        dev_label.append([])\n",
    "\n",
    "# 输出验证集第一个样本的文本和标签\n",
    "dev_str[0], dev_label[0]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('杭州五洲国际', ['B-city', 'E-city', 'B-poi', 'I-poi', 'I-poi', 'E-poi'])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 35
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据分析\n",
    "\n",
    "在读取数据集之后，接下来我们首先对文本进行基础统计。数据统计是对文本的角度进行统计，比如字符个数、文本长度等，这些统计规律会帮助我们后续的模型选择和实验步骤。下面我们演示了基础的字符统计，后续大家也可以考虑加入自己的统计。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-25T15:58:13.019994Z",
     "iopub.status.busy": "2022-09-25T15:58:13.019420Z",
     "iopub.status.idle": "2022-09-25T15:58:13.025104Z",
     "shell.execute_reply": "2022-09-25T15:58:13.023583Z",
     "shell.execute_reply.started": "2022-09-25T15:58:13.019946Z"
    }
   },
   "source": [
    "- 问题1：拥有多少个中文字？文本长度的规律是？"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.444397Z",
     "start_time": "2025-06-22T09:18:46.433922Z"
    }
   },
   "source": [
    "# 统计字符个数\n",
    "len(set(''.join(train_str)))"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2307"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 36
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:46.462112Z",
     "start_time": "2025-06-22T09:18:46.454403Z"
    }
   },
   "source": [
    "from scipy.stats import describe\n",
    "\n",
    "# 统计输入文本的长度统计值\n",
    "# 最长文本长度为68，最短是3，平均长度是17\n",
    "describe([\n",
    "    len(x) for x in train_str\n",
    "])"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=8856, minmax=(np.int64(3), np.int64(69)), mean=np.float64(17.157859078590786), variance=np.float64(32.80534308392209), skewness=np.float64(1.0429906237323388), kurtosis=np.float64(2.472641491590407))"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 37
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 问题2：标签是如何分布的？"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:50.033487Z",
     "start_time": "2025-06-22T09:18:46.471118Z"
    }
   },
   "source": [
    "# 将训练集标签和验证集标签进行拼接\n",
    "label_set = sum(train_label, []) + sum(dev_label, [])\n",
    "\n",
    "# 将标签进行去重\n",
    "label_set = list(set(label_set))\n",
    "# label_set.insert(0, 'O')\n",
    "label_set"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['E-devzone',\n",
       " 'I-city',\n",
       " 'S-intersection',\n",
       " 'I-subpoi',\n",
       " 'E-assist',\n",
       " 'B-district',\n",
       " 'I-cellno',\n",
       " 'I-poi',\n",
       " 'B-houseno',\n",
       " 'I-roadno',\n",
       " 'B-roadno',\n",
       " 'E-community',\n",
       " 'E-district',\n",
       " 'E-road',\n",
       " 'I-intersection',\n",
       " 'B-floorno',\n",
       " 'E-distance',\n",
       " 'I-road',\n",
       " 'I-distance',\n",
       " 'B-intersection',\n",
       " 'B-town',\n",
       " 'E-subpoi',\n",
       " 'I-community',\n",
       " 'I-village_group',\n",
       " 'S-community',\n",
       " 'B-village_group',\n",
       " 'I-district',\n",
       " 'B-cellno',\n",
       " 'S-poi',\n",
       " 'B-community',\n",
       " 'B-assist',\n",
       " 'E-prov',\n",
       " 'B-poi',\n",
       " 'E-poi',\n",
       " 'E-roadno',\n",
       " 'I-devzone',\n",
       " 'E-town',\n",
       " 'I-assist',\n",
       " 'I-prov',\n",
       " 'E-houseno',\n",
       " 'B-road',\n",
       " 'B-devzone',\n",
       " 'E-cellno',\n",
       " 'E-village_group',\n",
       " 'I-houseno',\n",
       " 'E-city',\n",
       " 'B-city',\n",
       " 'E-floorno',\n",
       " 'S-district',\n",
       " 'O',\n",
       " 'B-prov',\n",
       " 'E-intersection',\n",
       " 'B-distance',\n",
       " 'B-subpoi',\n",
       " 'I-floorno',\n",
       " 'I-town',\n",
       " 'S-assist']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 38
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:53.605410Z",
     "start_time": "2025-06-22T09:18:50.040469Z"
    }
   },
   "source": [
    "# 将训练集标签和验证集标签进行拼接\n",
    "label_set = sum(train_label, []) + sum(dev_label, [])\n",
    "\n",
    "# 将标签进行去重\n",
    "label_set = list(set(label_set))\n",
    "# label_set.insert(0, 'O')\n",
    "label_set"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['E-devzone',\n",
       " 'I-city',\n",
       " 'S-intersection',\n",
       " 'I-subpoi',\n",
       " 'E-assist',\n",
       " 'B-district',\n",
       " 'I-cellno',\n",
       " 'I-poi',\n",
       " 'B-houseno',\n",
       " 'I-roadno',\n",
       " 'B-roadno',\n",
       " 'E-community',\n",
       " 'E-district',\n",
       " 'E-road',\n",
       " 'I-intersection',\n",
       " 'B-floorno',\n",
       " 'E-distance',\n",
       " 'I-road',\n",
       " 'I-distance',\n",
       " 'B-intersection',\n",
       " 'B-town',\n",
       " 'E-subpoi',\n",
       " 'I-community',\n",
       " 'I-village_group',\n",
       " 'S-community',\n",
       " 'B-village_group',\n",
       " 'I-district',\n",
       " 'B-cellno',\n",
       " 'S-poi',\n",
       " 'B-community',\n",
       " 'B-assist',\n",
       " 'E-prov',\n",
       " 'B-poi',\n",
       " 'E-poi',\n",
       " 'E-roadno',\n",
       " 'I-devzone',\n",
       " 'E-town',\n",
       " 'I-assist',\n",
       " 'I-prov',\n",
       " 'E-houseno',\n",
       " 'B-road',\n",
       " 'B-devzone',\n",
       " 'E-cellno',\n",
       " 'E-village_group',\n",
       " 'I-houseno',\n",
       " 'E-city',\n",
       " 'B-city',\n",
       " 'E-floorno',\n",
       " 'S-district',\n",
       " 'O',\n",
       " 'B-prov',\n",
       " 'E-intersection',\n",
       " 'B-distance',\n",
       " 'B-subpoi',\n",
       " 'I-floorno',\n",
       " 'I-town',\n",
       " 'S-assist']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 39
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-22T09:18:57.074433Z",
     "start_time": "2025-06-22T09:18:53.625310Z"
    }
   },
   "source": [
    "# 将训练集标签和验证集标签进行拼接\n",
    "label_set = sum(train_label, []) + sum(dev_label, [])\n",
    "\n",
    "# 将标签进行去重\n",
    "label_set = list(set(label_set))\n",
    "# label_set.insert(0, 'O')\n",
    "label_set"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['E-devzone',\n",
       " 'I-city',\n",
       " 'S-intersection',\n",
       " 'I-subpoi',\n",
       " 'E-assist',\n",
       " 'B-district',\n",
       " 'I-cellno',\n",
       " 'I-poi',\n",
       " 'B-houseno',\n",
       " 'I-roadno',\n",
       " 'B-roadno',\n",
       " 'E-community',\n",
       " 'E-district',\n",
       " 'E-road',\n",
       " 'I-intersection',\n",
       " 'B-floorno',\n",
       " 'E-distance',\n",
       " 'I-road',\n",
       " 'I-distance',\n",
       " 'B-intersection',\n",
       " 'B-town',\n",
       " 'E-subpoi',\n",
       " 'I-community',\n",
       " 'I-village_group',\n",
       " 'S-community',\n",
       " 'B-village_group',\n",
       " 'I-district',\n",
       " 'B-cellno',\n",
       " 'S-poi',\n",
       " 'B-community',\n",
       " 'B-assist',\n",
       " 'E-prov',\n",
       " 'B-poi',\n",
       " 'E-poi',\n",
       " 'E-roadno',\n",
       " 'I-devzone',\n",
       " 'E-town',\n",
       " 'I-assist',\n",
       " 'I-prov',\n",
       " 'E-houseno',\n",
       " 'B-road',\n",
       " 'B-devzone',\n",
       " 'E-cellno',\n",
       " 'E-village_group',\n",
       " 'I-houseno',\n",
       " 'E-city',\n",
       " 'B-city',\n",
       " 'E-floorno',\n",
       " 'S-district',\n",
       " 'O',\n",
       " 'B-prov',\n",
       " 'E-intersection',\n",
       " 'B-distance',\n",
       " 'B-subpoi',\n",
       " 'I-floorno',\n",
       " 'I-town',\n",
       " 'S-assist']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 40
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 问题3：标签出现的频次？"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.429059Z",
     "start_time": "2025-06-22T09:18:57.081410Z"
    }
   },
   "source": [
    "from collections import Counter\n",
    "\n",
    "# 利用counter统计标签的频次\n",
    "c = Counter()\n",
    "c.update(sum(train_label, []))\n",
    "\n",
    "# 按照次序倒序进行展示\n",
    "c.most_common()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I-poi', 16657),\n",
       " ('I-road', 8095),\n",
       " ('I-roadno', 7739),\n",
       " ('B-poi', 6354),\n",
       " ('E-poi', 6354),\n",
       " ('B-district', 6124),\n",
       " ('E-district', 6124),\n",
       " ('I-district', 5842),\n",
       " ('B-road', 5711),\n",
       " ('E-road', 5711),\n",
       " ('I-town', 5270),\n",
       " ('B-city', 5118),\n",
       " ('E-city', 5118),\n",
       " ('I-city', 4262),\n",
       " ('B-prov', 4003),\n",
       " ('E-prov', 4003),\n",
       " ('B-town', 3883),\n",
       " ('E-town', 3883),\n",
       " ('B-roadno', 3716),\n",
       " ('E-roadno', 3716),\n",
       " ('I-devzone', 3714),\n",
       " ('I-prov', 3688),\n",
       " ('O', 3588),\n",
       " ('I-houseno', 2335),\n",
       " ('B-houseno', 2187),\n",
       " ('E-houseno', 2187),\n",
       " ('I-subpoi', 2018),\n",
       " ('I-community', 1708),\n",
       " ('B-community', 1571),\n",
       " ('E-community', 1571),\n",
       " ('B-subpoi', 1437),\n",
       " ('E-subpoi', 1437),\n",
       " ('B-devzone', 1020),\n",
       " ('E-devzone', 1020),\n",
       " ('B-floorno', 827),\n",
       " ('E-floorno', 827),\n",
       " ('I-cellno', 491),\n",
       " ('B-cellno', 413),\n",
       " ('E-cellno', 413),\n",
       " ('I-floorno', 357),\n",
       " ('B-assist', 228),\n",
       " ('E-assist', 228),\n",
       " ('S-assist', 187),\n",
       " ('B-village_group', 152),\n",
       " ('E-village_group', 152),\n",
       " ('B-intersection', 103),\n",
       " ('E-intersection', 103),\n",
       " ('I-village_group', 84),\n",
       " ('I-intersection', 82),\n",
       " ('I-distance', 48),\n",
       " ('B-distance', 28),\n",
       " ('E-distance', 28),\n",
       " ('S-intersection', 18),\n",
       " ('I-assist', 15),\n",
       " ('S-poi', 2)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 41
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.443076Z",
     "start_time": "2025-06-22T09:19:00.438063Z"
    }
   },
   "source": [
    "# 上述是标签的统计结果，接下来我们对标签进行编码\n",
    "\n",
    "# 从标签编码为数值\n",
    "tag_to_ix = {key: idx for idx, key in enumerate(label_set)}\n",
    "\n",
    "# 从数值还原为标签\n",
    "ix_to_tag = {idx: key for idx, key in enumerate(label_set)}\n",
    "\n",
    "tag_to_ix['E-assist'], ix_to_tag[40]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 'B-road')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 42
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 尝试1：LSTM实体识别\n",
    "\n",
    "### 思路介绍\n",
    "\n",
    "在进行初步的数据分析后，接下来我们将进行搭建模型。我们首先搭建LSTM进行实体识别的模型，其中底层为输入层，输入的字符在嵌入层转化为嵌入向量，前向LSTM和后向LSTM分别对序列进行表示，获取当前序列的上下文信息，构建序列的双向表达。\n",
    "\n",
    "![https://cdn.coggle.club/img/lstm-crf-tag.jpeg](https://cdn.coggle.club/img/lstm-crf-tag.jpeg?imageView2/4/w/500)\n",
    "\n",
    "在搭建这个模型时，需要注意以下事项：\n",
    "- 输入的文本如何处理，如何将文本进行编码？\n",
    "- 模型网络结构如何搭建，输入和输出如何对应？\n",
    "- 如何使用Pytorch搭建模型的损失函数和训练过程？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据处理"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.562176Z",
     "start_time": "2025-06-22T09:19:00.451081Z"
    }
   },
   "source": [
    "# 将序列进行编码\n",
    "# seq 输入的序列\n",
    "# to_ix 序列与标签的对应关系\n",
    "def prepare_sequence(seq, to_ix):\n",
    "    idxs = [to_ix[w] for w in seq]\n",
    "    return torch.tensor(idxs, dtype=torch.long)\n",
    "\n",
    "# 训练数据，输入文本和标签，一一对应的关系\n",
    "training_data = [[list(x), y] for x, y in zip(train_str, train_label)]\n",
    "\n",
    "# 验证数据，输入文本和标签，一一对应的关系\n",
    "dev_data = [[list(x), y] for x, y in zip(dev_str[:-1], dev_label[:-1])]\n",
    "\n",
    "# 文本字符进行编码，得到字符与数值的对应的关系\n",
    "word_to_ix = {}\n",
    "for sent, tags in (training_data + dev_data):\n",
    "    for word in sent:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)\n",
    "\n",
    "for line in test_lines:\n",
    "    line = line.strip()\n",
    "    for word in line:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)"
   ],
   "outputs": [],
   "execution_count": 43
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型定义"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.570766Z",
     "start_time": "2025-06-22T09:19:00.567258Z"
    }
   },
   "source": [
    "EMBEDDING_DIM = 128  # 文本嵌入的维度\n",
    "HIDDEN_DIM = 128  # LSTM 隐含层维度"
   ],
   "outputs": [],
   "execution_count": 44
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.585958Z",
     "start_time": "2025-06-22T09:19:00.579900Z"
    }
   },
   "source": [
    "# 定义LSTM 实体抽取模型\n",
    "class LSTMTagger(nn.Module):\n",
    "    def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size):\n",
    "        super(LSTMTagger, self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "        # 嵌入层，将文本进行嵌入\n",
    "        self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
    "\n",
    "        # 【修改点1】LSTM层，将文本提取特征, bidirectional=True 表示使用双向LSTM\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, bidirectional=True)\n",
    "\n",
    "        # 【修改点2】分类层，将特征转换为类别。因为是双向LSTM，所以输出维度是 hidden_dim * 2\n",
    "        self.hidden2tag = nn.Linear(hidden_dim * 2, tagset_size)\n",
    "        \n",
    "        # 将输出的概率进行归一化\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "\n",
    "    def forward(self, sentence):\n",
    "        embeds = self.word_embeddings(sentence)\n",
    "        lstm_out, _ = self.lstm(embeds.view(len(sentence), 1, -1))\n",
    "        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))\n",
    "        tag_scores = self.softmax(tag_space)\n",
    "        return tag_scores"
   ],
   "outputs": [],
   "execution_count": 45
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.600088Z",
     "start_time": "2025-06-22T09:19:00.590965Z"
    }
   },
   "source": [
    "# 定义模型\n",
    "model = LSTMTagger(EMBEDDING_DIM, HIDDEN_DIM, len(word_to_ix), len(tag_to_ix))\n",
    "model"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMTagger(\n",
       "  (word_embeddings): Embedding(3305, 128)\n",
       "  (lstm): LSTM(128, 128, bidirectional=True)\n",
       "  (hidden2tag): Linear(in_features=256, out_features=57, bias=True)\n",
       "  (softmax): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 46
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.607475Z",
     "start_time": "2025-06-22T09:19:00.604095Z"
    }
   },
   "source": [
    "# 模型超参数\n",
    "LR = 0.001 # 学习率\n",
    "EPOCH = 10# Epoch个数"
   ],
   "outputs": [],
   "execution_count": 47
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:19:00.616886Z",
     "start_time": "2025-06-22T09:19:00.613481Z"
    }
   },
   "source": [
    "# 定义损失函数\n",
    "loss_function = nn.NLLLoss()\n",
    "\n",
    "# 定义优化器\n",
    "optimizer = optim.Adam(model.parameters(), lr=LR)"
   ],
   "outputs": [],
   "execution_count": 48
  },
  {
   "cell_type": "code",
   "metadata": {
    "scrolled": true,
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:42:10.774502Z",
     "start_time": "2025-06-22T09:19:00.622891Z"
    }
   },
   "source": [
    "for epoch in range(EPOCH):\n",
    "    # 模型训练部分\n",
    "    for sentence, tags in training_data:\n",
    "        \n",
    "        # 模型梯度清空\n",
    "        model.zero_grad()\n",
    "        \n",
    "        # 对输入的句子进行编码\n",
    "        sentence_in = prepare_sequence(sentence, word_to_ix)\n",
    "        \n",
    "        # 对标签进行编码\n",
    "        targets = prepare_sequence(tags, tag_to_ix)\n",
    "        \n",
    "        # 输入句子得到模型的输出\n",
    "        tag_scores = model(sentence_in)\n",
    "        \n",
    "        # 计算模型输出与标签的概率差异性\n",
    "        loss = loss_function(tag_scores, targets)\n",
    "        \n",
    "        # 通过损失计算模型参数的梯度\n",
    "        loss.backward()\n",
    "        \n",
    "        # 模型的参数更新\n",
    "        optimizer.step()\n",
    "\n",
    "    # 验证部分\n",
    "    val_acc = []\n",
    "    with torch.no_grad():\n",
    "        for sentence, tags in dev_data[:]:\n",
    "            # 对输入的句子进行编码\n",
    "            sentence_in = prepare_sequence(sentence, word_to_ix)\n",
    "            \n",
    "            # 对标签进行编码\n",
    "            targets = prepare_sequence(tags, tag_to_ix)\n",
    "            \n",
    "            # 输入句子得到模型的输出\n",
    "            tag_scores = model(sentence_in)\n",
    "            \n",
    "            # 计算模型预测结果与真实标签的差异性，得到字符级别预测的准确率\n",
    "            val_acc.append((tag_scores.argmax(1) == targets).float().mean().item())\n",
    "\n",
    "    print('Epoch', epoch, np.mean(val_acc))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 0.7804214791768245\n",
      "Epoch 1 0.7913963126576491\n",
      "Epoch 2 0.794986806049716\n",
      "Epoch 3 0.8170333179196125\n",
      "Epoch 4 0.825694935080515\n",
      "Epoch 5 0.8260623294122631\n",
      "Epoch 6 0.8281310161660771\n",
      "Epoch 7 0.8291902384479639\n",
      "Epoch 8 0.8282580141904693\n",
      "Epoch 9 0.829274140381571\n"
     ]
    }
   ],
   "execution_count": 49
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型预测\n",
    "\n",
    "- 对测试集进行单条预测"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:42:10.906536Z",
     "start_time": "2025-06-22T09:42:10.878951Z"
    }
   },
   "source": [
    "# 读取测试集文本进行预测\n",
    "text = list(test_lines[25].strip().split('\\x01')[1])\n",
    "\n",
    "# 对标签进行编码\n",
    "sentence_in = prepare_sequence(text, word_to_ix)\n",
    "\n",
    "# 对测试集文本进行预测\n",
    "tag_scores = model(sentence_in)"
   ],
   "outputs": [],
   "execution_count": 50
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:42:10.927904Z",
     "start_time": "2025-06-22T09:42:10.915914Z"
    }
   },
   "source": [
    "# 测试集文本\n",
    "text"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['北', '京', '市', '西', '城', '区', '二', '龙', '路', '0', '0', '号']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 51
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-06-22T09:42:10.943817Z",
     "start_time": "2025-06-22T09:42:10.937856Z"
    }
   },
   "source": [
    "# 测试集预测结果\n",
    "[ix_to_tag[x] for x in tag_scores.argmax(1).numpy()]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['B-city',\n",
       " 'I-city',\n",
       " 'E-city',\n",
       " 'B-district',\n",
       " 'I-district',\n",
       " 'E-district',\n",
       " 'B-road',\n",
       " 'I-road',\n",
       " 'E-road',\n",
       " 'B-roadno',\n",
       " 'I-roadno',\n",
       " 'E-roadno']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 52
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 对测试集进行批量预测"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-22T09:44:00.648762Z",
     "start_time": "2025-06-22T09:42:10.980195Z"
    }
   },
   "source": [
    "with open('lstm_addr_parsing_runid.txt', 'w', encoding='utf8') as up:\n",
    "    for text in test_lines[:]:\n",
    "        idx = text.strip().split('\\x01')[0]\n",
    "        text = text.strip().split('\\x01')[1]\n",
    "\n",
    "        sentence_in = prepare_sequence(text, word_to_ix)\n",
    "\n",
    "        tag_scores = model(sentence_in)\n",
    "        tag_prediction = [ix_to_tag[x] for x in tag_scores.argmax(1).numpy()]\n",
    "        \n",
    "        up.write(\n",
    "            str(idx) + '\\u0001' + text + '\\u0001' + ' '.join(tag_prediction) + '\\n'\n",
    "        )"
   ],
   "outputs": [],
   "execution_count": 53
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "tianchi_metadata": {
   "competitions": [],
   "datasets": [
    {
     "id": "101024",
     "title": "获取数据集标题失败"
    }
   ],
   "description": "",
   "notebookId": "398335",
   "source": "dsw"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
